package com.cxl.mapreduce._11log_easy;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class LogDriver {
	//日志清洗：去除日志中字段长度小于等于11（根据 " " 来切分数据，切分后的块>11 ）的日志
	public static void main(String[] args) throws RuntimeException, Exception {

		// 输入输出路径需要根据自己电脑上实际的输入输出路径设置
		args = new String[] {  "/Users/a123/devWorkspace/bigData/hdfs/src/main/resources/input/web.log",
				"/Users/a123/devWorkspace/bigData/hdfs/src/main/resources/output/log_easy" };

		// 1 获取job信息
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);

		// 2 加载jar包
		job.setJarByClass(LogDriver.class);

		// 3 关联map
		job.setMapperClass(LogMapper.class);

		// 4 设置最终输出类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(NullWritable.class);

		// 设置reducetask个数为0
		job.setNumReduceTasks(0);

		// 5 设置输入和输出路径
		FileInputFormat.setInputPaths(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));

		// 6 提交
		job.waitForCompletion(true);

	}
}
